# Imports generales
import pandas as pd
import numpy as np
import seaborn as sns
import matplotlib.pyplot as plt
import scipy.stats as stats
import io
from google.colab import files

seed = 42  # Semilla aleatoria arbitraria y constante a incluir en los algoritmos estocásticos para que los experimentos sean siempre reproducibles por el profesor.
# OJO: En los experimentos estocásticos que requieran varias iteraciones con distintas semillas, podéis incorporarla como seed+1, seed+2, etc.

def upload_files (index_fields=None):
  uploaded = files.upload()
  for fn in uploaded.keys():
    print('User uploaded file "{name}" with length {length} bytes'.format(
        name=fn, length=len(uploaded[fn])))
    df = pd.read_csv(io.StringIO(uploaded[fn].decode('utf-8')), index_col = index_fields)
    return df

# Subir el conjunto de entrenamiento sin variable objetivo (dengue_features_train.csv)
train = upload_files()
print(train.shape)

Saving dengue_features_train.csv to dengue_features_train.csv
User uploaded file "dengue_features_train.csv" with length 287139 bytes
(1456, 24)

train.head()

train.tail()

plt.style.use('seaborn-v0_8-pastel')

fig, axs = plt.subplots(10, 2, figsize = (10, 15))

i, j = (0, 0)
for col in train.iloc[:, 4:].columns:
  agg = train.groupby('year')[col].mean()
  axs[i, j].plot(agg)
  axs[i, j].set_title(col)
  axs[i, j].tick_params(axis='x', labelsize=8)
  if j < 1:
    j += 1
  else:
    j = 0
    i += 1
fig.suptitle('Evolución media de las características entre 1990 y 2010')
plt.tight_layout()
plt.show()

fig, axs = plt.subplots(10, 2, figsize = (10, 15))

i, j = (0, 0)
for col in train.iloc[:, 4:].columns:
  agg = train.groupby('weekofyear')[col].mean()
  axs[i, j].plot(agg)
  axs[i, j].set_title(col)
  axs[i, j].tick_params(axis='x', labelsize=8)
  if j < 1:
    j += 1
  else:
    j = 0
    i += 1
fig.suptitle('Evolución media de las características anual')
plt.tight_layout()
plt.show()

train['weekofyear'].value_counts().plot(kind = 'bar')

<Axes: xlabel='weekofyear'>

train.drop(["year", "week_start_date"], axis = 1, inplace = True)
train = train[train['weekofyear'] < 53]

from sklearn.preprocessing import LabelEncoder

encoder = LabelEncoder()
train['city'] = encoder.fit_transform(train['city'])

<ipython-input-12-07c71286bdb0>:4: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame.
Try using .loc[row_indexer,col_indexer] = value instead

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy
  train['city'] = encoder.fit_transform(train['city'])

sns.pairplot(train, diag_kind = 'kde')
plt.show()

train.columns

Index(['city', 'weekofyear', 'ndvi_ne', 'ndvi_nw', 'ndvi_se', 'ndvi_sw',
       'precipitation_amt_mm', 'reanalysis_air_temp_k',
       'reanalysis_avg_temp_k', 'reanalysis_dew_point_temp_k',
       'reanalysis_max_air_temp_k', 'reanalysis_min_air_temp_k',
       'reanalysis_precip_amt_kg_per_m2',
       'reanalysis_relative_humidity_percent', 'reanalysis_sat_precip_amt_mm',
       'reanalysis_specific_humidity_g_per_kg', 'reanalysis_tdtr_k',
       'station_avg_temp_c', 'station_diur_temp_rng_c', 'station_max_temp_c',
       'station_min_temp_c', 'station_precip_mm'],
      dtype='object')

sns.pairplot(train[['ndvi_ne', 'ndvi_nw', 'ndvi_se', 'ndvi_sw']], diag_kind = 'kde')
plt.show()

sns.pairplot(train[['reanalysis_avg_temp_k', 'reanalysis_dew_point_temp_k',
       'reanalysis_max_air_temp_k', 'reanalysis_min_air_temp_k',
       'reanalysis_precip_amt_kg_per_m2',
       'reanalysis_relative_humidity_percent', 'reanalysis_sat_precip_amt_mm',
       'reanalysis_specific_humidity_g_per_kg', 'reanalysis_tdtr_k']], diag_kind = 'kde')
plt.show()

sns.pairplot(train[['station_avg_temp_c', 'station_diur_temp_rng_c', 'station_max_temp_c',
       'station_min_temp_c', 'station_precip_mm']], diag_kind = 'kde')
plt.show()

def plot_heatmap(df, labels, max = 1, min = -1):
  #Se genera una máscara por el triángulo superior
  mask = np.triu(np.ones_like(df, dtype=bool))

  #Se establece la figura de matplotlib
  f, ax = plt.subplots(figsize=(11, 11))

  #Se genera un mapa de color divergente
  cmap = sns.diverging_palette(230, 20, as_cmap=True)

  #Se plotea el mapa de calor con la máscara y un ratio correcto
  sns.heatmap(df, mask=mask, cmap=cmap, vmax=max, vmin = min, center=0,
              xticklabels = labels, yticklabels = labels,
              square=True, linewidths=.5, cbar_kws={"shrink": .5},
              annot = True, fmt=".2f", annot_kws = {"size": 8})
  plt.xticks(fontsize = 8)
  plt.yticks(fontsize = 8)

#Código adaptado de https://seaborn.pydata.org/examples/many_pairwise_correlations.html

#Se genera un dataframe de correlaciones
train_corr = train.corr()
labels = train.columns.tolist()
plot_heatmap(train_corr, labels)

sns.pairplot(train[['reanalysis_sat_precip_amt_mm', 'precipitation_amt_mm']],
             diag_kind = 'kde')
plt.show()

sns.pairplot(train[['reanalysis_specific_humidity_g_per_kg',
                    'reanalysis_dew_point_temp_k']],
             diag_kind = 'kde')
plt.show()

sns.pairplot(train[['reanalysis_tdtr_k',
                    'station_diur_temp_rng_c',
                    'reanalysis_max_air_temp_k']],
             diag_kind = 'kde')
plt.show()

train.drop(['precipitation_amt_mm', 'reanalysis_dew_point_temp_k'], axis = 1, inplace = True)

train.describe()

train.shape

(1451, 20)

def porc_nulls(train):
  for col in train.columns:
    nulos = len(train[train[col].isnull()])
    porc_nulos = round((nulos / train.shape[0]) * 100, 0)
    print(f"Valores nulos en {col}: {porc_nulos} %")

porc_nulls(train)

#La función ha sido creada porque se llamará de nuevo más adelante, así evito duplicar código

Valores nulos en city: 0.0 %
Valores nulos en weekofyear: 0.0 %
Valores nulos en ndvi_ne: 13.0 %
Valores nulos en ndvi_nw: 3.0 %
Valores nulos en ndvi_se: 1.0 %
Valores nulos en ndvi_sw: 1.0 %
Valores nulos en reanalysis_air_temp_k: 0.0 %
Valores nulos en reanalysis_avg_temp_k: 0.0 %
Valores nulos en reanalysis_max_air_temp_k: 0.0 %
Valores nulos en reanalysis_min_air_temp_k: 0.0 %
Valores nulos en reanalysis_precip_amt_kg_per_m2: 0.0 %
Valores nulos en reanalysis_relative_humidity_percent: 0.0 %
Valores nulos en reanalysis_sat_precip_amt_mm: 1.0 %
Valores nulos en reanalysis_specific_humidity_g_per_kg: 0.0 %
Valores nulos en reanalysis_tdtr_k: 0.0 %
Valores nulos en station_avg_temp_c: 3.0 %
Valores nulos en station_diur_temp_rng_c: 3.0 %
Valores nulos en station_max_temp_c: 1.0 %
Valores nulos en station_min_temp_c: 1.0 %
Valores nulos en station_precip_mm: 1.0 %

'''Se genera un df con los booleanos True o False de na, es decir, cada valor na
será sustituido por True, sino, por False'''
train_na = train.isna()
#Se genera una serie para usar de filtro con tan solo las filas con algun na
filter_na = train.isna().any(axis = 1)
'''Ahora, se calcula el ratio entre columnas con na y el total de columnas.
Como criterio, se establecerá que, en caso de que haya un 50% o más de
características nulas, se eliminará la instancia.'''
ind_high_na = [] #Se crea una lista para almacenar indices de ins. a elim.
for i, row in train_na[filter_na].iterrows():
  ratio_na = row.sum() / row.size
  if ratio_na >= 0.5:
    ind_high_na.append(i)

#Ahora, se eliminan dichos indices del dataset
train.drop(index = ind_high_na, inplace = True)

porc_nulls(train)

Valores nulos en city: 0.0 %
Valores nulos en weekofyear: 0.0 %
Valores nulos en ndvi_ne: 13.0 %
Valores nulos en ndvi_nw: 3.0 %
Valores nulos en ndvi_se: 1.0 %
Valores nulos en ndvi_sw: 1.0 %
Valores nulos en reanalysis_air_temp_k: 0.0 %
Valores nulos en reanalysis_avg_temp_k: 0.0 %
Valores nulos en reanalysis_max_air_temp_k: 0.0 %
Valores nulos en reanalysis_min_air_temp_k: 0.0 %
Valores nulos en reanalysis_precip_amt_kg_per_m2: 0.0 %
Valores nulos en reanalysis_relative_humidity_percent: 0.0 %
Valores nulos en reanalysis_sat_precip_amt_mm: 0.0 %
Valores nulos en reanalysis_specific_humidity_g_per_kg: 0.0 %
Valores nulos en reanalysis_tdtr_k: 0.0 %
Valores nulos en station_avg_temp_c: 2.0 %
Valores nulos en station_diur_temp_rng_c: 2.0 %
Valores nulos en station_max_temp_c: 1.0 %
Valores nulos en station_min_temp_c: 0.0 %
Valores nulos en station_precip_mm: 1.0 %

train.shape

(1446, 20)

train.info()

<class 'pandas.core.frame.DataFrame'>
Index: 1446 entries, 0 to 1455
Data columns (total 20 columns):
 #   Column                                 Non-Null Count  Dtype  
---  ------                                 --------------  -----  
 0   city                                   1446 non-null   int64  
 1   weekofyear                             1446 non-null   int64  
 2   ndvi_ne                                1257 non-null   float64
 3   ndvi_nw                                1399 non-null   float64
 4   ndvi_se                                1429 non-null   float64
 5   ndvi_sw                                1429 non-null   float64
 6   reanalysis_air_temp_k                  1446 non-null   float64
 7   reanalysis_avg_temp_k                  1446 non-null   float64
 8   reanalysis_max_air_temp_k              1446 non-null   float64
 9   reanalysis_min_air_temp_k              1446 non-null   float64
 10  reanalysis_precip_amt_kg_per_m2        1446 non-null   float64
 11  reanalysis_relative_humidity_percent   1446 non-null   float64
 12  reanalysis_sat_precip_amt_mm           1443 non-null   float64
 13  reanalysis_specific_humidity_g_per_kg  1446 non-null   float64
 14  reanalysis_tdtr_k                      1446 non-null   float64
 15  station_avg_temp_c                     1413 non-null   float64
 16  station_diur_temp_rng_c                1413 non-null   float64
 17  station_max_temp_c                     1436 non-null   float64
 18  station_min_temp_c                     1442 non-null   float64
 19  station_precip_mm                      1434 non-null   float64
dtypes: float64(18), int64(2)
memory usage: 237.2 KB

sesgos = {} #En este diccionario se almacenarán los sesgos por característica
tipos_sesgos = ('Bajo sesgo o aproximadamente simétrica',
                'Moderadamente sesgada',
                'Altamente sesgada')
for col in train.columns:
  sesgo = stats.skew(train[col], nan_policy = 'omit')
  if -0.5 < sesgo < 0.5:
        tipo = tipos_sesgos[0]
  elif -1 < sesgo <= -0.5 or 0.5 <= sesgo < 1:
      tipo = tipos_sesgos[1]
  elif sesgo <= -1 or sesgo >= 1:
      tipo = tipos_sesgos[2]

  sesgos[col] = tipo
  print(f"Característica: {col}; Sesgo: {round(sesgo, 2)}, {tipo}")

#Referencia: https://docs.scipy.org/doc/scipy/reference/generated/scipy.stats.skew.html

Característica: city; Sesgo: -0.6, Moderadamente sesgada
Característica: weekofyear; Sesgo: 0.0, Bajo sesgo o aproximadamente simétrica
Característica: ndvi_ne; Sesgo: -0.1, Bajo sesgo o aproximadamente simétrica
Característica: ndvi_nw; Sesgo: -0.01, Bajo sesgo o aproximadamente simétrica
Característica: ndvi_se; Sesgo: 0.57, Moderadamente sesgada
Característica: ndvi_sw; Sesgo: 0.75, Moderadamente sesgada
Característica: reanalysis_air_temp_k; Sesgo: -0.08, Bajo sesgo o aproximadamente simétrica
Característica: reanalysis_avg_temp_k; Sesgo: -0.19, Bajo sesgo o aproximadamente simétrica
Característica: reanalysis_max_air_temp_k; Sesgo: 0.85, Moderadamente sesgada
Característica: reanalysis_min_air_temp_k; Sesgo: -0.67, Moderadamente sesgada
Característica: reanalysis_precip_amt_kg_per_m2; Sesgo: 3.38, Altamente sesgada
Característica: reanalysis_relative_humidity_percent; Sesgo: 0.57, Moderadamente sesgada
Característica: reanalysis_sat_precip_amt_mm; Sesgo: 1.74, Altamente sesgada
Característica: reanalysis_specific_humidity_g_per_kg; Sesgo: -0.54, Moderadamente sesgada
Característica: reanalysis_tdtr_k; Sesgo: 1.07, Altamente sesgada
Característica: station_avg_temp_c; Sesgo: -0.57, Moderadamente sesgada
Característica: station_diur_temp_rng_c; Sesgo: 0.84, Moderadamente sesgada
Característica: station_max_temp_c; Sesgo: -0.26, Bajo sesgo o aproximadamente simétrica
Característica: station_min_temp_c; Sesgo: -0.31, Bajo sesgo o aproximadamente simétrica
Característica: station_precip_mm; Sesgo: 2.98, Altamente sesgada

for col in train.columns:
  if col == 'ndvi_ne':
    continue
  if sesgos[col] == tipos_sesgos[0]:
    #train[col].fillna(train[col].mean(), inplace = True) #Obsoleto a partir de pandas 3.0, me apareció mensaje de error
    train.fillna({col: train[col].mean()}, inplace = True)
  else:
    #train[col].fillna(train[col].median(), inplace = True) #Obsoleto a partir de pandas 3.0, me apareció mensaje de error
    train.fillna({col: train[col].median()}, inplace = True)

porc_nulls(train)

Valores nulos en city: 0.0 %
Valores nulos en weekofyear: 0.0 %
Valores nulos en ndvi_ne: 13.0 %
Valores nulos en ndvi_nw: 0.0 %
Valores nulos en ndvi_se: 0.0 %
Valores nulos en ndvi_sw: 0.0 %
Valores nulos en reanalysis_air_temp_k: 0.0 %
Valores nulos en reanalysis_avg_temp_k: 0.0 %
Valores nulos en reanalysis_max_air_temp_k: 0.0 %
Valores nulos en reanalysis_min_air_temp_k: 0.0 %
Valores nulos en reanalysis_precip_amt_kg_per_m2: 0.0 %
Valores nulos en reanalysis_relative_humidity_percent: 0.0 %
Valores nulos en reanalysis_sat_precip_amt_mm: 0.0 %
Valores nulos en reanalysis_specific_humidity_g_per_kg: 0.0 %
Valores nulos en reanalysis_tdtr_k: 0.0 %
Valores nulos en station_avg_temp_c: 0.0 %
Valores nulos en station_diur_temp_rng_c: 0.0 %
Valores nulos en station_max_temp_c: 0.0 %
Valores nulos en station_min_temp_c: 0.0 %
Valores nulos en station_precip_mm: 0.0 %

from sklearn.linear_model import LinearRegression

df_to_predict = train[train.isna().any(axis = 1)][['ndvi_ne', 'ndvi_nw', 'ndvi_se', 'ndvi_sw']]
df_subtrain = train[~train.isna().any(axis = 1)][['ndvi_ne', 'ndvi_nw', 'ndvi_se', 'ndvi_sw']]

from sklearn.model_selection import train_test_split
#https://scikit-learn.org/stable/modules/generated/sklearn.model_selection.train_test_split.html

X = df_subtrain[['ndvi_nw', 'ndvi_se', 'ndvi_sw']] #Variables independientes o explicativas
y = df_subtrain[['ndvi_ne']] #Variable dependiente o explicada

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2,
                                                    random_state = 42,
                                                    shuffle = True)

linear_model = LinearRegression()
linear_model.fit(X_train, y_train)
#https://scikit-learn.org/dev/modules/generated/sklearn.linear_model.LinearRegression.html

LinearRegression()

LinearRegression()

linear_model.score(X_test, y_test)

0.727325236857673

df_to_predict = train[train.isna().any(axis = 1)]
df_subtrain = train[~train.isna().any(axis = 1)]
X = df_subtrain.drop('ndvi_ne', axis = 1) #Variables independientes o explicativas
y = df_subtrain[['ndvi_ne']] #Variable dependiente o explicada
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.2,
                                                    random_state = 42,
                                                    shuffle = True)
linear_model = LinearRegression()
linear_model.fit(X_train, y_train)
linear_model.score(X_test, y_test)

0.7399935557180093

X = df_to_predict.drop('ndvi_ne', axis = 1)

y_pred = linear_model.predict(X)

dict_index_values = {}
j = 0
for i, _ in df_to_predict['ndvi_ne'].items():
  dict_index_values[i] = y_pred[j]
  j += 1

train.fillna({'ndvi_ne': dict_index_values}, inplace = True)

train['ndvi_ne'] = train['ndvi_ne'].astype(float)

porc_nulls(train)

Valores nulos en city: 0.0 %
Valores nulos en weekofyear: 0.0 %
Valores nulos en ndvi_ne: 0.0 %
Valores nulos en ndvi_nw: 0.0 %
Valores nulos en ndvi_se: 0.0 %
Valores nulos en ndvi_sw: 0.0 %
Valores nulos en reanalysis_air_temp_k: 0.0 %
Valores nulos en reanalysis_avg_temp_k: 0.0 %
Valores nulos en reanalysis_max_air_temp_k: 0.0 %
Valores nulos en reanalysis_min_air_temp_k: 0.0 %
Valores nulos en reanalysis_precip_amt_kg_per_m2: 0.0 %
Valores nulos en reanalysis_relative_humidity_percent: 0.0 %
Valores nulos en reanalysis_sat_precip_amt_mm: 0.0 %
Valores nulos en reanalysis_specific_humidity_g_per_kg: 0.0 %
Valores nulos en reanalysis_tdtr_k: 0.0 %
Valores nulos en station_avg_temp_c: 0.0 %
Valores nulos en station_diur_temp_rng_c: 0.0 %
Valores nulos en station_max_temp_c: 0.0 %
Valores nulos en station_min_temp_c: 0.0 %
Valores nulos en station_precip_mm: 0.0 %

train.duplicated().any()

False

sesgos

{'city': 'Moderadamente sesgada',
 'weekofyear': 'Bajo sesgo o aproximadamente simétrica',
 'ndvi_ne': 'Bajo sesgo o aproximadamente simétrica',
 'ndvi_nw': 'Bajo sesgo o aproximadamente simétrica',
 'ndvi_se': 'Moderadamente sesgada',
 'ndvi_sw': 'Moderadamente sesgada',
 'reanalysis_air_temp_k': 'Bajo sesgo o aproximadamente simétrica',
 'reanalysis_avg_temp_k': 'Bajo sesgo o aproximadamente simétrica',
 'reanalysis_max_air_temp_k': 'Moderadamente sesgada',
 'reanalysis_min_air_temp_k': 'Moderadamente sesgada',
 'reanalysis_precip_amt_kg_per_m2': 'Altamente sesgada',
 'reanalysis_relative_humidity_percent': 'Moderadamente sesgada',
 'reanalysis_sat_precip_amt_mm': 'Altamente sesgada',
 'reanalysis_specific_humidity_g_per_kg': 'Moderadamente sesgada',
 'reanalysis_tdtr_k': 'Altamente sesgada',
 'station_avg_temp_c': 'Moderadamente sesgada',
 'station_diur_temp_rng_c': 'Moderadamente sesgada',
 'station_max_temp_c': 'Bajo sesgo o aproximadamente simétrica',
 'station_min_temp_c': 'Bajo sesgo o aproximadamente simétrica',
 'station_precip_mm': 'Altamente sesgada'}

def increment_index(k):
  for i in range (k):
    for j in range (k):
      yield i, j

#Referencia: https://ellibrodepython.com/yield-python

import math
def plotear_boxplots(tipo):
  global train
  global sesgos
  cols_to_plot = []

  for col, value in sesgos.items():
    if value == tipo:
      #plt.boxplot(train[col])
      cols_to_plot.append(col)

  '''Como quiero crear una serie de subplots y que se pinten en un grid
  cuadrado, calculo la raíz cuadrada k (entera redondeada hacia arriba) de la
  cantidad de gráficas a pintar. El resultado será una matriz de subplots de kxk'''
  k = math.ceil(math.sqrt(len(cols_to_plot)))

  #Se crea el generador para el valor de k como límite del grid
  incr_i_j = increment_index(k)

  #Se crea el grid
  fig, axs = plt.subplots(k, k, figsize = (10, 10))
  plt.suptitle(f'Boxplots de las características de tipo: {tipo}')

  for col in cols_to_plot:
    i, j = next(incr_i_j)
    axs[i, j].boxplot(train[col], patch_artist = True,
                      boxprops={'facecolor': 'skyblue'},
                      medianprops={'color': 'blue', 'linewidth': 1.5,
                                  'linestyle': '--'},
                      flierprops = {'markerfacecolor': 'red',
                                    'markersize': 4,
                                    'linestyle': 'none'})
    axs[i, j].set_title(f'{col}', fontsize = 8)
    axs[i, j].set_xticks([])

  plt.tight_layout(rect = [0, 0, 1, 0.96])
  plt.show()

#Referencias:
  #https://matplotlib.org/stable/gallery/subplots_axes_and_figures/subplots_demo.html
  #https://matplotlib.org/3.1.1/api/_as_gen/matplotlib.pyplot.boxplot.html
  #chatgpt

for tipo in tipos_sesgos:
  plotear_boxplots(tipo)

train['reanalysis_precip_amt_kg_per_m2'] = train['reanalysis_precip_amt_kg_per_m2'].apply(
    lambda x: x if x < 320 else train['reanalysis_precip_amt_kg_per_m2'].mean())

train['station_precip_mm'] = train['station_precip_mm'].apply(
    lambda x: x if x < 320 else train['station_precip_mm'].mean())

train['reanalysis_sat_precip_amt_mm'] = train['reanalysis_sat_precip_amt_mm'].apply(
    lambda x: x if x < 260 else train['reanalysis_sat_precip_amt_mm'].mean())

plotear_boxplots(tipos_sesgos[2])

# NDVIs
## Media ndvi
train['calc_ndvi_avg'] = train[['ndvi_ne', 'ndvi_nw',
                                    'ndvi_se', 'ndvi_sw']].mean(axis = 1)

## Rangos ndvi
train['calc_ndvi_range'] = train[['ndvi_ne', 'ndvi_nw',
                 'ndvi_se', 'ndvi_sw']].max(axis = 1)-train[['ndvi_ne',
                  'ndvi_nw', 'ndvi_se', 'ndvi_sw']].min(axis = 1)

# Temperaturas
## temp_range
train['calc_temp_range_c'] = train['station_max_temp_c'] - train['station_min_temp_c']

## total_avg_air_temp
train['calc_total_avg_air_temp'] = (train['reanalysis_air_temp_k'] - 273.15 + train['station_avg_temp_c']) / 2

## temp_range_k
train['calc_temp_range_k'] = train['reanalysis_max_air_temp_k'] - train['reanalysis_min_air_temp_k']

## avg_temp_range
train['calc_avg_temp_range_c'] = (train['calc_temp_range_k'] - 273.15 + train['calc_temp_range_c']) / 2

# Precipitaciones
## avg_precip
train['calc_avg_precip'] = train[['reanalysis_precip_amt_kg_per_m2', 'station_precip_mm']].mean(axis=1)

#Referencias:
#https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.max.html

train.columns

Index(['city', 'weekofyear', 'ndvi_ne', 'ndvi_nw', 'ndvi_se', 'ndvi_sw',
       'reanalysis_air_temp_k', 'reanalysis_avg_temp_k',
       'reanalysis_max_air_temp_k', 'reanalysis_min_air_temp_k',
       'reanalysis_precip_amt_kg_per_m2',
       'reanalysis_relative_humidity_percent', 'reanalysis_sat_precip_amt_mm',
       'reanalysis_specific_humidity_g_per_kg', 'reanalysis_tdtr_k',
       'station_avg_temp_c', 'station_diur_temp_rng_c', 'station_max_temp_c',
       'station_min_temp_c', 'station_precip_mm', 'calc_ndvi_avg',
       'calc_ndvi_range', 'calc_temp_range_c', 'calc_total_avg_air_temp',
       'calc_temp_range_k', 'calc_avg_temp_range_c', 'calc_avg_precip'],
      dtype='object')

train_corr = train.corr()
plot_heatmap(train_corr, train.columns.tolist())

#Código adaptado de https://seaborn.pydata.org/examples/many_pairwise_correlations.html

train.drop(['calc_temp_range_k', 'calc_avg_temp_range_c'],
           axis = 1, inplace = True)

train_transposed = train.T
train_transposed.head()

names = train_transposed.index
names

Index(['city', 'weekofyear', 'ndvi_ne', 'ndvi_nw', 'ndvi_se', 'ndvi_sw',
       'reanalysis_air_temp_k', 'reanalysis_avg_temp_k',
       'reanalysis_max_air_temp_k', 'reanalysis_min_air_temp_k',
       'reanalysis_precip_amt_kg_per_m2',
       'reanalysis_relative_humidity_percent', 'reanalysis_sat_precip_amt_mm',
       'reanalysis_specific_humidity_g_per_kg', 'reanalysis_tdtr_k',
       'station_avg_temp_c', 'station_diur_temp_rng_c', 'station_max_temp_c',
       'station_min_temp_c', 'station_precip_mm', 'calc_ndvi_avg',
       'calc_ndvi_range', 'calc_temp_range_c', 'calc_total_avg_air_temp',
       'calc_avg_precip'],
      dtype='object')

#http://scikit-learn.org/stable/modules/preprocessing.html
from sklearn.preprocessing import MinMaxScaler
min_max_scaler = MinMaxScaler()
features_norm = min_max_scaler.fit_transform(train_transposed)

#http://docs.scipy.org/doc/scipy/reference/cluster.html
from scipy import cluster
from sklearn import metrics
dist = metrics.DistanceMetric.get_metric('euclidean')
matdist = dist.pairwise(features_norm)

matdist_scaled = min_max_scaler.fit_transform(matdist.flatten().reshape(-1, 1))
matdist_scaled = matdist_scaled.reshape(matdist.shape[0], matdist.shape[1])

#Se usa una función definida anteriormente para mantener formato en heatmaps
plot_heatmap(matdist_scaled, names, max = 1, min = 0)#Se usa una función definida anteriormente para mantener formato en heatmaps

from scipy import cluster

# http://docs.scipy.org/doc/scipy/reference/generated/scipy.cluster.hierarchy.linkage.html#scipy.cluster.hierarchy.linkage
clusters = cluster.hierarchy.linkage(matdist, method = 'single')
# http://docs.scipy.org/doc/scipy-0.14.0/reference/generated/scipy.cluster.hierarchy.dendrogram.html
k = 2
fig, axs = plt.subplots(k, k, figsize = (10, 10))
#Utilizo el generador creado anteriormente para subplots cuadrados
incr_i_j = increment_index(k)
for cut in (5, 10, 15, 20):
  i, j = next(incr_i_j)
  cluster.hierarchy.dendrogram(clusters, color_threshold = cut, leaf_font_size = 8,
                                labels = names , leaf_rotation=90, ax = axs[i, j])
  axs[i, j].set_title(f'Cut: {cut}', fontsize = 8)
plt.show()

<ipython-input-66-8619e0d2c8ac>:2: ClusterWarning: The symmetric non-negative hollow observation matrix looks suspiciously like an uncondensed distance matrix
  clusters = cluster.hierarchy.linkage(matdist, method = 'single')

from sklearn.decomposition import PCA
pca = PCA (n_components = 2)
X_pca = pca.fit_transform(features_norm)
print("Componentes lineales:\n", pca.components_)
print("\nRatio de variabilidad: ", pca.explained_variance_ratio_, "\n")

Componentes lineales:
 [[ 0.02673872  0.02674034  0.02662135 ...  0.02609486  0.02594988
   0.02594301]
 [-0.01404983 -0.01882628 -0.00384252 ... -0.00368385  0.00231955
  -0.00321332]]

Ratio de variabilidad:  [0.97857145 0.0132881 ]

cut = 10
labels = cluster.hierarchy.fcluster(clusters, cut , criterion = 'distance')
labels

array([2, 2, 2, 2, 2, 2, 1, 1, 1, 1, 3, 4, 3, 2, 2, 2, 2, 2, 2, 3, 2, 2,
       2, 2, 3], dtype=int32)

plt.scatter(X_pca[:,0], X_pca[:,1], c=labels, s=50, cmap="Set2")
for i in range(len(X_pca)):
    plt.text(X_pca[i][0], X_pca[i][1], names[i], fontsize = 8)

plt.show()

xmax = 0
xmin = -10
ymax = -0.5
ymin = -1
for i in range(len(X_pca)):
    if (X_pca[i][0] > xmin and
        X_pca[i][1] < xmax and
        X_pca[i][1] > ymin and
        X_pca[i][1] < ymax):
      plt.scatter(X_pca[i][0], X_pca[i][1], c=labels[i],s=50, cmap="Set2")
      plt.text(X_pca[i][0], X_pca[i][1], names[i], fontsize = 8)

plt.show()

from sklearn.cluster import KMeans

train

train_norm = min_max_scaler.fit_transform(train)

distortions = []
silhouettes = []

for i in range(2, 20):
    km = KMeans(i, init='k-means++', n_init=10, random_state=42)
    clustering = km.fit_predict(train_norm)
    distortions.append(km.inertia_)
    silhouettes.append(metrics.silhouette_score(train_norm, clustering))

plt.plot(range(2,20), distortions, marker='o')
plt.xticks(range(2, 20))
plt.xlabel('K')
plt.ylabel('Distortion')
plt.grid(True)
plt.show()

!pip install kneed
import kneed

Collecting kneed
  Downloading kneed-0.8.5-py3-none-any.whl.metadata (5.5 kB)
Requirement already satisfied: numpy>=1.14.2 in /usr/local/lib/python3.10/dist-packages (from kneed) (1.26.4)
Requirement already satisfied: scipy>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from kneed) (1.13.1)
Downloading kneed-0.8.5-py3-none-any.whl (10 kB)
Installing collected packages: kneed
Successfully installed kneed-0.8.5

kneedle = kneed.KneeLocator(range(2, 20), distortions[:20], curve="convex",
                            direction="decreasing")
elbow_point = kneedle.elbow
print('Elbow: ', elbow_point)
kneedle.plot_knee()

Elbow:  6

def plot_silhouettes(silhouettes, min, max, xLabel, step = 1):
  plt.plot(range(min, max, step), silhouettes , marker='o')
  plt.xticks(range(min, max, step))
  plt.xlabel(xLabel)
  plt.ylabel('Silhouette')
  plt.tick_params(axis = 'x', labelsize = 8)
  plt.grid(True)
  plt.show()
  print("Valores de silueta:")
  print([round(silhouette, 2) for silhouette in silhouettes])

plot_silhouettes(silhouettes, 2, 20, 'K Clusters')

Valores de silueta:
[0.47, 0.38, 0.31, 0.28, 0.28, 0.28, 0.22, 0.21, 0.21, 0.2, 0.2, 0.15, 0.16, 0.15, 0.15, 0.14, 0.14, 0.13]

X_pca = pca.fit_transform(train_norm)

def plot_train_norm_2D(clustering, algorithm_type, size = 10):
  plt.scatter(X_pca[:, 0], X_pca[:, 1], s=size, c = clustering, cmap="Set3",
              zorder = 0)
  plt.title(f"Clustering con {algorithm_type} en 2D")
  plt.xlabel("Componente Principal 1")
  plt.ylabel("Componente Principal 2")
  plt.grid(True)
  plt.show()

#Referencia: apuntes de clase

km = KMeans(2, init = 'k-means++', n_init = 10, random_state  = 42)
clustering = km.fit_predict(train_norm)
centroids_2d = pca.fit_transform(km.cluster_centers_)
plt.scatter(centroids_2d[:, 0], centroids_2d[:, 1], marker='x', s=100, c='red')
plot_train_norm_2D(clustering, 'K-Means')

km = KMeans(elbow_point, init = 'k-means++', n_init = 10, random_state  = 42)
clustering = km.fit_predict(train_norm)
centroids_2d = pca.fit_transform(km.cluster_centers_)
plt.scatter(centroids_2d[:, 0], centroids_2d[:, 1], marker='x', s=100, c='red')
plot_train_norm_2D(clustering, 'K-Means')

from sklearn.cluster import DBSCAN

silhouettes = []
porc_noise_points = []

for i in range(2, 40):
    dbscan = DBSCAN(eps=0.5, min_samples = i)
    clustering = dbscan.fit_predict(train_norm)
    silhouettes.append(metrics.silhouette_score(train_norm, clustering))
    noise_points_index = np.where(clustering == -1)[0]
    noise_points = train_norm[noise_points_index]
    porc_n_p = round(noise_points.shape[0] / train_norm.shape[0], 2)
    porc_noise_points.append(porc_n_p)

plot_silhouettes(silhouettes, 2, 40, 'min_samples')

Valores de silueta:
[0.11, 0.33, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.4, 0.39, 0.39, 0.39, 0.38, 0.38, 0.38, 0.38, 0.37, 0.37, 0.37, 0.37, 0.37, 0.37, 0.36, 0.36, 0.36, 0.36, 0.36, 0.35, 0.35, 0.35, 0.34, 0.34, 0.34, 0.33, 0.33, 0.33, 0.33]

plt.plot(range(2,40), porc_noise_points , marker='o')
plt.xticks(range(2, 40))
plt.xlabel('min_samples')
plt.ylabel('% Puntos de Ruido')
plt.tick_params(axis = 'x', labelsize = 5)
plt.grid(True)
plt.show()

param_grid = {
    'eps': np.arange(0.2, 1, 0.1),
    'min_samples': range(2, 40)
}

best_sil_score = 0
best_params = {key: 0 for key in param_grid.keys()}
for value1 in param_grid['eps']:
  for value2 in param_grid['min_samples']:
    dbscan = DBSCAN(eps=value1, min_samples = value2)
    clustering = dbscan.fit_predict(train_norm)
    n_labels = len(np.unique(clustering))
    if n_labels == 1:  #Esto se hace para evitar un error de silhouete_score
      continue
    sil_score = metrics.silhouette_score(train_norm, clustering)
    if sil_score > best_sil_score:
      best_sil_score = sil_score
      best_params['eps'] = value1
      best_params['min_samples'] = value2
      noise_points_index = np.where(clustering == -1)[0]
      noise_points = train_norm[noise_points_index]
      porc_n_p = round(noise_points.shape[0] / train_norm.shape[0], 2)

print("Mejores hiperparámetros:", best_params)
print("Mejor puntuación de la silueta:", best_sil_score)
print("Ratio de puntos de ruido:", porc_n_p)

Mejores hiperparámetros: {'eps': 0.9000000000000001, 'min_samples': 2}
Mejor puntuación de la silueta: 0.4671721515440993
Ratio de puntos de ruido: 0.0

dbscan = DBSCAN(eps=0.9, min_samples = 2)
clustering = dbscan.fit_predict(train_norm)

np.unique(clustering)

array([0, 1])

plot_train_norm_2D(clustering, 'DBSCAN')

from sklearn.mixture import GaussianMixture

silhouettes = []
for i in range(2, 20):
    gmm = GaussianMixture(n_components=i)
    clustering = gmm.fit_predict(train_norm)
    silhouettes.append(metrics.silhouette_score(train_norm, clustering))

plot_silhouettes(silhouettes, 2, 20, 'GMM')

Valores de silueta:
[0.47, 0.35, 0.32, 0.23, 0.24, 0.16, 0.23, 0.11, 0.14, 0.16, 0.11, 0.11, 0.09, 0.12, 0.08, 0.09, 0.09, 0.08]

gmm = GaussianMixture(n_components=2)
clustering = gmm.fit_predict(train_norm)
plot_train_norm_2D(clustering, 'GMM')
metrics.silhouette_score(train_norm, clustering)

0.4671721515440993

probs = gmm.predict_proba(train_norm)
probs[:30]

array([[0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.],
       [0., 1.]])

np.unique(probs)

array([0., 1.])

from matplotlib.patches import Ellipse

def draw_ellipse(position, covariance, ax=None, **kwargs):
    """Draw an ellipse with a given position and covariance"""
    ax = ax or plt.gca()

    # Convert covariance to principal axes
    if covariance.shape == (2, 2):
        U, s, Vt = np.linalg.svd(covariance)
        angle = np.degrees(np.arctan2(U[1, 0], U[0, 0]))
        width, height = 2 * np.sqrt(s)
    else:
        angle = 0
        width, height = 2 * np.sqrt(covariance)

    # Draw the Ellipse
    for nsig in range(1, 4):
        ax.add_patch(Ellipse(position, nsig * width, nsig * height,
                             angle, **kwargs))

def plot_gmm(gmm, X, label=True, ax=None):
    ax = ax or plt.gca()
    labels = gmm.fit(X).predict(X)
    if label:
        ax.scatter(X[:, 0], X[:, 1], c=labels, s=10, cmap='viridis', zorder=2)
    else:
        ax.scatter(X[:, 0], X[:, 1], s=10, zorder=2)
    ax.axis('equal')

    w_factor = 0.2 / gmm.weights_.max()
    for pos, covar, w in zip(gmm.means_, gmm.covariances_, gmm.weights_):
        draw_ellipse(pos, covar, alpha=w * w_factor)

    ax.scatter(gmm.means_[:, 0], gmm.means_[:, 1], marker='*', c='r', s=300, zorder=3)

#Referencia: apuntes aportados en clase

plot_gmm(gmm, X_pca)
plt.grid(True)
plt.show()

<ipython-input-99-7ca1e15ecad3>:18: MatplotlibDeprecationWarning: Passing the angle parameter of __init__() positionally is deprecated since Matplotlib 3.6; the parameter will become keyword-only two minor releases later.
  ax.add_patch(Ellipse(position, nsig * width, nsig * height,

n_components = np.arange(2, 20)
models = [GaussianMixture(n, covariance_type='full', random_state=42)
          for n in n_components]
aics = [model.fit(train_norm).aic(train_norm) for model in models]
plt.plot(n_components, aics, marker = 'o')
plt.grid(True)
#Referencia: apuntes de clase

gmm = GaussianMixture(n_components=8)
clustering = gmm.fit_predict(train_norm)
plot_train_norm_2D(clustering, 'GMM')
metrics.silhouette_score(train_norm, clustering)

0.20193210651635332

fig, ax1 = plt.subplots()

ax1.plot(range(2, 20), silhouettes , marker='o',
         color = 'salmon', label = 'Silhouette')
ax1.set_xticks(range(2, 20))
ax1.set_ylabel('Silhouette')
ax2 = ax1.twinx()
ax2.plot(n_components, aics, marker = 'o',
               color = 'skyblue', label = 'AIC')
ax2.set_ylabel('AIC')
ax1.legend(loc = 'upper right')
ax2.legend(loc = 'upper left')
plt.axvline(x = 3, color = 'black', linestyle = '--')
plt.grid(True)
plt.tight_layout()
plt.show()

from sklearn.cluster import AffinityPropagation

silhouettes = []
for i in range(-110, 0, 10):
    af = AffinityPropagation(preference = i, random_state=42,
                             max_iter = 1000, convergence_iter = 30)
    clustering = af.fit_predict(train_norm)
    if len(np.unique(clustering)) == train_norm.shape[0]:
      silhouettes.append(0)
    else:
      silhouettes.append(metrics.silhouette_score(train_norm, clustering))

/usr/local/lib/python3.10/dist-packages/sklearn/cluster/_affinity_propagation.py:142: ConvergenceWarning: Affinity propagation did not converge, this model may return degenerate cluster centers and labels.
  warnings.warn(

plot_silhouettes(silhouettes, -110, 0, 'Affinity', 10)

Valores de silueta:
[0.38, 0, 0.31, 0.38, 0.31, 0.29, 0.31, 0.31, 0.28, 0.26, 0.16]

af = AffinityPropagation(preference = -80, random_state=42,
                         max_iter = 1000, convergence_iter = 30)
clustering = af.fit_predict(train_norm)

np.unique(clustering)

array([0, 1, 2])

plot_train_norm_2D(clustering, 'AP')
metrics.silhouette_score(train_norm, clustering)

0.37815382611842574

gmm = GaussianMixture(n_components=3)
clustering = gmm.fit_predict(train_norm)
plot_train_norm_2D(clustering, 'GMM')
metrics.silhouette_score(train_norm, clustering)

0.3522048606563886

train_norm = pd.DataFrame(train_norm, columns = train.columns)
train_norm['cluster'] = clustering
train_norm.to_csv('train_norm_clustered.csv', index = True)

train_norm.head()

files.download('train_norm_clustered.csv')

	city	year	weekofyear	week_start_date	ndvi_ne	ndvi_nw	ndvi_se	ndvi_sw	precipitation_amt_mm	reanalysis_air_temp_k	...	reanalysis_precip_amt_kg_per_m2	reanalysis_relative_humidity_percent	reanalysis_sat_precip_amt_mm	reanalysis_specific_humidity_g_per_kg	reanalysis_tdtr_k	station_avg_temp_c	station_diur_temp_rng_c	station_max_temp_c	station_min_temp_c	station_precip_mm
0	sj	1990	18	1990-04-30	0.122600	0.103725	0.198483	0.177617	12.42	297.572857	...	32.00	73.365714	12.42	14.012857	2.628571	25.442857	6.900000	29.4	20.0	16.0
1	sj	1990	19	1990-05-07	0.169900	0.142175	0.162357	0.155486	22.82	298.211429	...	17.94	77.368571	22.82	15.372857	2.371429	26.714286	6.371429	31.7	22.2	8.6
2	sj	1990	20	1990-05-14	0.032250	0.172967	0.157200	0.170843	34.54	298.781429	...	26.10	82.052857	34.54	16.848571	2.300000	26.714286	6.485714	32.2	22.8	41.4
3	sj	1990	21	1990-05-21	0.128633	0.245067	0.227557	0.235886	15.36	298.987143	...	13.90	80.337143	15.36	16.672857	2.428571	27.471429	6.771429	33.3	23.3	4.0
4	sj	1990	22	1990-05-28	0.196200	0.262200	0.251200	0.247340	7.52	299.518571	...	12.20	80.460000	7.52	17.210000	3.014286	28.942857	9.371429	35.0	23.9	5.8

	city	year	weekofyear	week_start_date	ndvi_ne	ndvi_nw	ndvi_se	ndvi_sw	precipitation_amt_mm	reanalysis_air_temp_k	...	reanalysis_precip_amt_kg_per_m2	reanalysis_relative_humidity_percent	reanalysis_sat_precip_amt_mm	reanalysis_specific_humidity_g_per_kg	reanalysis_tdtr_k	station_avg_temp_c	station_diur_temp_rng_c	station_max_temp_c	station_min_temp_c	station_precip_mm
1451	iq	2010	21	2010-05-28	0.342750	0.318900	0.256343	0.292514	55.30	299.334286	...	45.00	88.765714	55.30	18.485714	9.800000	28.633333	11.933333	35.4	22.4	27.0
1452	iq	2010	22	2010-06-04	0.160157	0.160371	0.136043	0.225657	86.47	298.330000	...	207.10	91.600000	86.47	18.070000	7.471429	27.433333	10.500000	34.7	21.7	36.6
1453	iq	2010	23	2010-06-11	0.247057	0.146057	0.250357	0.233714	58.94	296.598571	...	50.60	94.280000	58.94	17.008571	7.500000	24.400000	6.900000	32.2	19.2	7.4
1454	iq	2010	24	2010-06-18	0.333914	0.245771	0.278886	0.325486	59.67	296.345714	...	62.33	94.660000	59.67	16.815714	7.871429	25.433333	8.733333	31.2	21.0	16.0
1455	iq	2010	25	2010-06-25	0.298186	0.232971	0.274214	0.315757	63.22	298.097143	...	36.90	89.082857	63.22	17.355714	11.014286	27.475000	9.900000	33.7	22.2	20.4

	city	weekofyear	ndvi_ne	ndvi_nw	ndvi_se	ndvi_sw	reanalysis_air_temp_k	reanalysis_avg_temp_k	reanalysis_max_air_temp_k	reanalysis_min_air_temp_k	reanalysis_precip_amt_kg_per_m2	reanalysis_relative_humidity_percent	reanalysis_sat_precip_amt_mm	reanalysis_specific_humidity_g_per_kg	reanalysis_tdtr_k	station_avg_temp_c	station_diur_temp_rng_c	station_max_temp_c	station_min_temp_c	station_precip_mm
count	1451.000000	1451.000000	1262.000000	1404.000000	1434.000000	1434.000000	1446.000000	1446.000000	1446.000000	1446.000000	1446.000000	1446.000000	1443.000000	1446.000000	1446.000000	1413.000000	1413.000000	1436.000000	1442.000000	1434.000000
mean	0.643005	26.412130	0.142294	0.130553	0.203783	0.202305	298.701852	299.225578	303.427109	295.719156	40.151819	82.161959	45.760388	16.746427	4.903754	27.185783	8.059328	32.452437	22.102150	39.326360
std	0.479279	14.964361	0.140531	0.119999	0.073860	0.083903	1.362420	1.261715	3.234601	2.565364	43.434399	7.153897	43.715537	1.542494	3.546445	1.292347	2.128568	1.959318	1.574066	47.455314
min	0.000000	1.000000	-0.406250	-0.456100	-0.015533	-0.063457	294.635714	294.892857	297.800000	286.900000	0.000000	57.787143	0.000000	11.715714	1.357143	21.400000	4.528571	26.700000	14.700000	0.000000
25%	0.000000	13.000000	0.044950	0.049217	0.155087	0.144209	297.658929	298.257143	301.000000	293.900000	13.055000	77.177143	9.800000	15.557143	2.328571	26.300000	6.514286	31.100000	21.100000	8.700000
50%	1.000000	26.000000	0.128817	0.121429	0.196050	0.189450	298.646429	299.289286	302.400000	296.200000	27.245000	80.301429	38.340000	17.087143	2.857143	27.414286	7.300000	32.800000	22.200000	23.850000
75%	1.000000	39.000000	0.248483	0.216600	0.248846	0.246982	299.833571	300.207143	305.500000	297.900000	52.200000	86.357857	70.235000	17.978214	7.625000	28.157143	9.566667	33.900000	23.300000	53.900000
max	1.000000	52.000000	0.508357	0.454429	0.538314	0.546017	302.200000	302.928571	314.000000	299.900000	570.500000	98.610000	390.600000	20.461429	16.028571	30.800000	15.800000	42.200000	25.600000	543.300000

	0	1	2	3	4	5	6	7	8	9	...	1446	1447	1448	1449	1450	1451	1452	1453	1454	1455
city	1.000000	1.000000	1.000000	1.000000	1.0000	1.000000	1.000000	1.000000	1.000000	1.000000	...	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000	0.000000
weekofyear	18.000000	19.000000	20.000000	21.000000	22.0000	23.000000	24.000000	25.000000	26.000000	27.000000	...	16.000000	17.000000	18.000000	19.000000	20.000000	21.000000	22.000000	23.000000	24.000000	25.000000
ndvi_ne	0.122600	0.169900	0.032250	0.128633	0.1962	0.164221	0.112900	0.072500	0.102450	0.093376	...	0.231486	0.239743	0.260814	0.168686	0.263071	0.342750	0.160157	0.247057	0.333914	0.298186
ndvi_nw	0.103725	0.142175	0.172967	0.245067	0.2622	0.174850	0.092800	0.072500	0.146175	0.121550	...	0.294686	0.259271	0.255786	0.158500	0.272500	0.318900	0.160371	0.146057	0.245771	0.232971
ndvi_se	0.198483	0.162357	0.157200	0.227557	0.2512	0.254314	0.205071	0.151471	0.125571	0.160683	...	0.331657	0.307786	0.257771	0.133071	0.258271	0.256343	0.136043	0.250357	0.278886	0.274214

	city	weekofyear	ndvi_ne	ndvi_nw	ndvi_se	ndvi_sw	reanalysis_air_temp_k	reanalysis_avg_temp_k	reanalysis_max_air_temp_k	reanalysis_min_air_temp_k	...	station_avg_temp_c	station_diur_temp_rng_c	station_max_temp_c	station_min_temp_c	station_precip_mm	calc_ndvi_avg	calc_ndvi_range	calc_temp_range_c	calc_total_avg_air_temp	calc_avg_precip
0	1	18	0.122600	0.103725	0.198483	0.177617	297.572857	297.742857	299.8	295.9	...	25.442857	6.900000	29.4	20.0	16.0	0.150606	0.094758	9.4	24.932857	24.000
1	1	19	0.169900	0.142175	0.162357	0.155486	298.211429	298.442857	300.9	296.4	...	26.714286	6.371429	31.7	22.2	8.6	0.157479	0.027725	9.5	25.887857	13.270
2	1	20	0.032250	0.172967	0.157200	0.170843	298.781429	298.878571	300.5	297.3	...	26.714286	6.485714	32.2	22.8	41.4	0.133315	0.140717	9.4	26.172857	33.750
3	1	21	0.128633	0.245067	0.227557	0.235886	298.987143	299.228571	301.4	297.0	...	27.471429	6.771429	33.3	23.3	4.0	0.209286	0.116433	10.0	26.654286	8.950
4	1	22	0.196200	0.262200	0.251200	0.247340	299.518571	299.664286	301.9	297.5	...	28.942857	9.371429	35.0	23.9	5.8	0.239235	0.066000	11.1	27.655714	9.000
...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...	...
1451	0	21	0.342750	0.318900	0.256343	0.292514	299.334286	300.771429	309.7	294.5	...	28.633333	11.933333	35.4	22.4	27.0	0.302627	0.086407	13.0	27.408810	36.000
1452	0	22	0.160157	0.160371	0.136043	0.225657	298.330000	299.392857	308.5	291.9	...	27.433333	10.500000	34.7	21.7	36.6	0.170557	0.089614	13.0	26.306667	121.850
1453	0	23	0.247057	0.146057	0.250357	0.233714	296.598571	297.592857	305.5	292.4	...	24.400000	6.900000	32.2	19.2	7.4	0.219296	0.104300	13.0	23.924286	29.000
1454	0	24	0.333914	0.245771	0.278886	0.325486	296.345714	297.521429	306.1	291.9	...	25.433333	8.733333	31.2	21.0	16.0	0.296014	0.088143	10.2	24.314524	39.165
1455	0	25	0.298186	0.232971	0.274214	0.315757	298.097143	299.835714	307.8	292.3	...	27.475000	9.900000	33.7	22.2	20.4	0.280282	0.082786	11.5	26.211071	28.650

PRÁCTICA 1: APRENDIZAJE NO SUPERVISADO (DengAI)¶

Resumen¶

Ingeniería de características¶

Clusterización¶

Inicialización¶

Exploración¶

Aclaración del significado de las variables¶

Características NDVI: Índice De Vegetación De Diferencia Normalizada¶

NOAA's NCEP Climate Forecast System Reanalysis measurements, NOAA's GHCN daily climate data weather station measurements y¶

Codificación de características categóricas¶

Correlación de características¶

Estadísticas generales¶

Tratamiento de nulos¶

Análisis de la simetría¶

Cumplimentación de características con bajo % de nulos¶

Cumplimentación de la característica con alto % de nulos: Modelo de Regresión Lineal¶

Tratamiento de duplicados¶

Outliers¶

Características¶

Creación de nuevas características¶

1. Promedio y Rango de NDVI¶

2. Índices de Temperaturas¶

3. Precipitación media¶

Relación Jerárquica de Características¶

Normalización: MinMaxScaler¶

Matriz de similitud¶

Jerarquía mediante Linkage¶

PCA (Principal Component Analysis)¶

Clustering¶

K-MEANS¶

DBSCAN¶

EXPECTATION - MAXIMIZATION: Gaussian Mixture Models¶

AFFINITY PROPAGATION¶

Conclusiones¶

	city	weekofyear	ndvi_ne	ndvi_nw	ndvi_se	ndvi_sw	reanalysis_air_temp_k	reanalysis_avg_temp_k	reanalysis_max_air_temp_k	reanalysis_min_air_temp_k	...	station_diur_temp_rng_c	station_max_temp_c	station_min_temp_c	station_precip_mm	calc_ndvi_avg	calc_ndvi_range	calc_temp_range_c	calc_total_avg_air_temp	calc_avg_precip	cluster
0	1.0	0.333333	0.578226	0.614835	0.386418	0.395544	0.388291	0.354667	0.123457	0.692308	...	0.210393	0.174194	0.486239	0.052305	0.407631	0.141899	0.261745	0.358874	0.099174	2
1	1.0	0.352941	0.629943	0.657063	0.321190	0.359233	0.472710	0.441778	0.191358	0.730769	...	0.163498	0.322581	0.688073	0.028114	0.419152	0.032783	0.268456	0.504184	0.054835	2
2	1.0	0.372549	0.479441	0.690881	0.311879	0.384430	0.548064	0.496000	0.166667	0.800000	...	0.173638	0.354839	0.743119	0.135338	0.378645	0.216710	0.261745	0.547549	0.139463	1
3	1.0	0.392157	0.584823	0.770066	0.438912	0.491150	0.575260	0.539556	0.222222	0.776923	...	0.198986	0.425806	0.788991	0.013076	0.505996	0.177182	0.302013	0.620802	0.036983	1
4	1.0	0.411765	0.658698	0.788882	0.481601	0.509943	0.645515	0.593778	0.253086	0.815385	...	0.429658	0.535484	0.844037	0.018960	0.556200	0.095086	0.375839	0.773177	0.037190	1